{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zqiao\\Downloads\\pyhealth\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "root_dir = os.getcwd().split('\\examples\\learning_models')[0]\n",
    "print (root_dir)\n",
    "os.chdir(root_dir)\n",
    "import sys\n",
    "sys.path.append(root_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current ExpData_ID: 2020.0810.data.phenotyping.mimic --- Target for MIMIC\n",
      "load finished\n",
      "target Task: phenotyping\n",
      "N of features: 59\n",
      "N of labels: 25\n",
      "N of TrainData: 76\n",
      "N of ValidData: 20\n",
      "N of TestData: 24\n",
      "------------Train--------------\n",
      "x_data ['./datasets/mimic\\\\x_data\\\\41976_125449.csv', './datasets/mimic\\\\x_data\\\\10126_160445.csv', './datasets/mimic\\\\x_data\\\\42231_171878.csv']\n",
      "y_data [array([1., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n",
      "       0., 0., 0., 0., 1., 0., 0., 0.]), array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,\n",
      "       0., 0., 1., 0., 1., 0., 1., 0.]), array([0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0.,\n",
      "       0., 0., 0., 0., 0., 0., 1., 0.])]\n",
      "l_data [9, 140, 4]\n",
      "------------Valid--------------\n",
      "x_data ['./datasets/mimic\\\\x_data\\\\10132_197611.csv', './datasets/mimic\\\\x_data\\\\10044_124073.csv', './datasets/mimic\\\\x_data\\\\41983_107689.csv']\n",
      "y_data [array([1., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 1., 0., 0., 0.,\n",
      "       0., 0., 0., 0., 0., 0., 1., 0.]), array([0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0.,\n",
      "       0., 0., 0., 0., 0., 0., 0., 0.]), array([0., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "       0., 0., 0., 0., 0., 0., 0., 0.])]\n",
      "l_data [2, 17, 7]\n",
      "------------Test--------------\n",
      "x_data ['./datasets/mimic\\\\x_data\\\\40503_168803.csv', './datasets/mimic\\\\x_data\\\\40277_127703.csv', './datasets/mimic\\\\x_data\\\\42367_139932.csv']\n",
      "y_data [array([0., 1., 0., 1., 1., 1., 0., 1., 1., 0., 1., 1., 0., 0., 0., 0., 0.,\n",
      "       0., 0., 0., 0., 0., 0., 0., 1.]), array([0., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.,\n",
      "       0., 1., 0., 0., 0., 0., 0., 0.]), array([0., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,\n",
      "       0., 0., 0., 0., 0., 0., 0., 0.])]\n",
      "l_data [3, 5, 116]\n"
     ]
    }
   ],
   "source": [
    "from pyhealth.data.expdata_generator import sequencedata as expdata_generator\n",
    "expdata_id = '2020.0810.data.phenotyping.mimic'\n",
    "# phenotyping\n",
    "cur_dataset = expdata_generator(expdata_id)\n",
    "# cur_dataset.get_exp_data(sel_task = 'phenotyping', data_root = r'./datasets/mimic')\n",
    "cur_dataset.load_exp_data()\n",
    "cur_dataset.show_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "  0%|                                                                                          | 0/100 [00:00<?, ?it/s]\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "not find effcient GPU, use CPU recource\n",
      "current task can beed seen as multilabel; loss func L1LossSigmoid is used for optimization\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "tr=>epoch=0 Valid Loss: 62.940, Train Loss: 63.097:   0%|                                      | 0/100 [00:02<?, ?it/s]\u001b[A\u001b[A\n",
      "\n",
      "tr=>epoch=0 Valid Loss: 62.940, Train Loss: 63.097:   1%|▎                             | 1/100 [00:02<03:23,  2.06s/it]\u001b[A\u001b[A"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-7c0ce9239245>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[0mexpmodel_id\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'2020.0820.gpu.test.phenotyping.cpu'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[0mclf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mexpmodel_id\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mexpmodel_id\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0muse_gpu\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m \u001b[0mclf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcur_dataset\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcur_dataset\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Downloads\\pyhealth\\pyhealth\\models\\sequence\\dipole.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, train_data, valid_data, assign_task_type)\u001b[0m\n\u001b[0;32m    431\u001b[0m         \u001b[0mtrain_reader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_reader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'train'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    432\u001b[0m         \u001b[0mvalid_reader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_reader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalid_data\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'valid'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 433\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_reader\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalid_reader\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    434\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    435\u001b[0m     def load_model(self, \n",
      "\u001b[1;32m~\\Downloads\\pyhealth\\pyhealth\\models\\sequence\\_dlbase.py\u001b[0m in \u001b[0;36m_fit_model\u001b[1;34m(self, train_reader, valid_reader)\u001b[0m\n\u001b[0;32m    368\u001b[0m         \u001b[0mtqdm_trange\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mn_epoch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    369\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtqdm_trange\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 370\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_train_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_reader\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    371\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_valid_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalid_reader\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    372\u001b[0m             \u001b[0mtrain_loss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0macc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'train'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Downloads\\pyhealth\\pyhealth\\models\\sequence\\_dlbase.py\u001b[0m in \u001b[0;36m_train_model\u001b[1;34m(self, train_loader)\u001b[0m\n\u001b[0;32m    273\u001b[0m             \u001b[0mtimetick\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimetick\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    274\u001b[0m             \u001b[0mdata_input\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m'X'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'cur_M'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mcur_masks\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'M'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mmasks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'T'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mtimetick\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 275\u001b[1;33m             \u001b[0mall_h\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mh\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredictor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata_input\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    276\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtarget_repl\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    277\u001b[0m                 \u001b[0mdata_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m'all_hat_y'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mall_h\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'hat_y'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mh\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'y'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mtargets\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'mask'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mmasks\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    548\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    549\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 550\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    551\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    552\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Downloads\\pyhealth\\pyhealth\\models\\sequence\\dipole.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input_data)\u001b[0m\n\u001b[0;32m    234\u001b[0m         \u001b[0mall_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict_func\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_batchsize\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mn_timestep\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutput_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    235\u001b[0m                          \u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_batchsize\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_timestep\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlabel_size\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mM\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 236\u001b[1;33m         \u001b[0mcur_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mall_output\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mcur_M\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    237\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mall_output\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcur_output\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    238\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from pyhealth.models.sequence.dipole import Dipole as model\n",
    "# from pyhealth.models.sequence.lstm import LSTM as model\n",
    "# from pyhealth.models.sequence.gru import GRU as model\n",
    "# from pyhealth.models.sequence.embedgru import EmbedGRU as model\n",
    "# from pyhealth.models.sequence.retain import Retain as model\n",
    "# from pyhealth.models.sequence.raim import RAIM as model\n",
    "# from pyhealth.models.sequence.tlstm import tLSTM as model\n",
    "# from pyhealth.models.sequence.xgboost import XGBoost as model\n",
    "# from pyhealth.models.sequence.rf import RandomForest as model\n",
    "\n",
    "expmodel_id = '2020.0820.gpu.test.phenotyping.cpu'\n",
    "clf = model(expmodel_id = expmodel_id, use_gpu = True)\n",
    "clf.fit(cur_dataset.train, cur_dataset.valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load predictor config file from ./experiments_records\\2020.0820.gpu.test.phenotyping.cpu\\checkouts\\predictor_config.json\n",
      "load best-th epoch model\n",
      "current task can beed seen as multilabel; loss func L1LossSigmoid is used for optimization\n",
      "current data evaluate using multilabel evaluation-type\n",
      "{'avg_precision_micro': 0.21915044940412448, 'roc_auc_score_micro': 0.4995143745143745, 'coverage_error': 21.05, 'label_ranking_average_precision_score': 0.34556477108006534, 'label_ranking_loss': 0.47491986612271975, 'hamming_loss@1': 0.236, 'recall@1': nan, 'precision@1': 0.15, 'hamming_loss@3': 0.276, 'recall@3': nan, 'precision@3': 0.2166666666666667}\n"
     ]
    }
   ],
   "source": [
    "clf.load_model()\n",
    "clf.inference(cur_dataset.test)\n",
    "results = clf.get_results()\n",
    "# print (results)\n",
    "\n",
    "from pyhealth.evaluation.evaluator import func \n",
    "r = func(results['hat_y'], results['y'])\n",
    "print (r)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
